{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 7.1 Breakoutを実行してみる\n",
    "\n",
    "Windowsでbreakout-v0を実行するために、事前に以下のコマンドを実行してください。\n",
    "\n",
    "pip install --no-index -f https://github.com/Kojoley/atari-py/releases atari_py\n",
    "\n",
    "pip install opencv-python\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# パッケージのimport\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import gym\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ゲームの開始\n",
    "ENV = 'Breakout-v0'  # 使用する課題名\n",
    "env = gym.make(ENV)  # 実行する課題を設定\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Box(210, 160, 3)\n",
      "Discrete(4)\n",
      "['NOOP', 'FIRE', 'RIGHT', 'LEFT']\n"
     ]
    }
   ],
   "source": [
    "# ゲームの状態と行動を把握\n",
    "\n",
    "# 状態\n",
    "print(env.observation_space)\n",
    "# Box(210, 160, 3)\n",
    "\n",
    "# 行動\n",
    "print(env.action_space)\n",
    "print(env.unwrapped.get_action_meanings())\n",
    "# Discrete(4)\n",
    "# ['NOOP', 'FIRE', 'RIGHT', 'LEFT']、 0: 何もしない、1: 玉を発射、2：右へ移動、3: 左へ移動\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1728d2b5e10>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANEAAAD8CAYAAADpCEEHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADntJREFUeJzt3X/sVfV9x/Hna1hNRruI9UcM4ABH2+myUUscmdN0c7VImqJL2kGWyjYzNJGkjS4Z1mQjS5psXcGk2UaDkRQXC7pRK1mshbCmZtmwgkWEIgqU1q8QmLiIw6YOeO+P8/mm1y/fy/dy3+f2nnt9PZKbe+/nnnPP+wRefM49nPu+igjMrHu/1O8CzAadQ2SW5BCZJTlEZkkOkVmSQ2SW1LMQSZovaZ+k/ZKW92o7Zv2mXvw/kaRJwMvAJ4AR4DlgcUT8sPaNmfVZr2ai64H9EXEwIt4BNgALe7Qts766oEfvOxV4teX5CPDb7RaW5MsmrIlej4jLJlqoVyHSOGPvCoqkpcDSHm3frA4/7mShXoVoBJje8nwacLh1gYhYA6wBz0Q22Hr1meg5YLakmZIuBBYBm3q0LbO+6slMFBGnJC0DvgNMAtZGxJ5ebMus33pyivu8i2jg4dyqVavOe51777039R5j16/rPbKaUMNYY2vq0TZ3RMTciRbyFQtmSb06sTB0ejFL9GO2q8MvYqYZJJ6JzJI8E9l5m2j2e6/NVJ6JzJI8E9mEJppZ+vG5rEk8E5kleSbqUB3/2jblPQZhm4PEM5FZkkNkluTLfsza82U/Zr8IjTixMG3atPfcf9BZ83X6d9IzkVmSQ2SW5BCZJTlEZkldh0jSdEnflbRX0h5Jny/jKyS9JmlnuS2or1yz5smcnTsF3BcRz0v6ALBD0pby2oMR8ZV8eWbN13WIIuIIcKQ8fkvSXqqmjWbvKbV8JpI0A/go8GwZWiZpl6S1kqbUsQ2zpkqHSNL7gY3AFyLiBLAauBqYQzVTrWyz3lJJ2yVtP3nyZLYMs75JhUjS+6gC9GhEfBMgIo5GxOmIOAM8RNXc/iwRsSYi5kbE3MmTJ2fKMOurzNk5AQ8DeyNiVcv4lS2L3Q7s7r48s+bLnJ27Afgc8KKknWXsi8BiSXOoGtgfAu5KVWjWcJmzc//B+L/+8FT35ZgNHl+xYJbUiK9CTMRfk7BeqKt3hGcisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLSn+fSNIh4C3gNHAqIuZKugR4DJhB9RXxz0bE/2S3ZdZEdc1EvxcRc1p+VWw5sDUiZgNby3OzodSrw7mFwLryeB1wW4+2Y9Z3dYQogM2SdkhaWsauKG2GR9sNX17DdswaqY4eCzdExGFJlwNbJL3UyUolcEsBpkxxp2EbXOmZKCIOl/tjwBNUHU+PjjZxLPfHxlnPHVBtKGTbCE8uP6uCpMnALVQdTzcBS8piS4AnM9sxa7Ls4dwVwBNVR2EuAL4REU9Leg54XNKdwE+AzyS3Y9ZYqRBFxEHgt8YZPw7cnHlvs0HhKxbMkgaiA+q2+fP7XYINof+s6X08E5klOURmSQ6RWZJDZJbkEJklDcTZuTO/dqLfJZi15ZnILMkhMktyiMySHCKzJIfILMkhMksaiFPcb/zK2/0uwawtz0RmSQ6RWVLXh3OSPkzV5XTULOCvgIuBPwf+u4x/MSKe6rpCs4brOkQRsQ+YAyBpEvAaVbefPwUejIiv1FKhWcPVdTh3M3AgIn5c0/uZDYy6zs4tAta3PF8m6Q5gO3Bftpn9Gx95J7O62fher+dt0jORpAuBTwP/UoZWA1dTHeodAVa2WW+ppO2Stp88eTJbhlnf1HE4dyvwfEQcBYiIoxFxOiLOAA9RdUQ9izug2rCoI0SLaTmUG20fXNxO1RHVbGilPhNJ+mXgE8BdLcNfljSH6tciDo15zWzoZDugvg18cMzY51IVmQ2Ygbh27htnrup3CTaEbqnpfXzZj1mSQ2SW5BCZJTlEZkkOkVnSQJyde2fDin6XYMPolnp+XMUzkVmSQ2SW5BCZJTlEZkkOkVmSQ2SWNBCnuP/96Xn9LsGG0KduWVXL+3gmMktyiMySHCKzpI5CJGmtpGOSdreMXSJpi6RXyv2UMi5JX5W0X9IuSdf1qnizJuh0Jvo6MH/M2HJga0TMBraW51B1/5ldbkupWmiZDa2OQhQRzwBvjBleCKwrj9cBt7WMPxKVbcDFYzoAmQ2VzGeiKyLiCEC5v7yMTwVebVlupIy9i5s32rDoxYkFjTMWZw24eaMNiUyIjo4eppX7Y2V8BJjestw04HBiO2aNlgnRJmBJebwEeLJl/I5ylm4e8OboYZ/ZMOrosh9J64GPA5dKGgH+Gvhb4HFJdwI/AT5TFn8KWADsB96m+r0is6HVUYgiYnGbl24eZ9kA7skUZTZIfMWCWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWZJDZJbkEJklOURmSQ6RWdKEIWrT/fTvJb1UOpw+IeniMj5D0k8l7Sy3r/WyeLMm6GQm+jpndz/dAvxGRPwm8DJwf8trByJiTrndXU+ZZs01YYjG634aEZsj4lR5uo2qLZbZe1Idn4n+DPh2y/OZkn4g6XuSbmy3kjug2rBI/VKepAeAU8CjZegIcFVEHJf0MeBbkq6NiBNj142INcAagOnTp5/VIdVsUHQ9E0laAnwK+OPSJouI+FlEHC+PdwAHgA/VUahZU3UVIknzgb8EPh0Rb7eMXyZpUnk8i+rnVQ7WUahZU014ONem++n9wEXAFkkA28qZuJuAv5F0CjgN3B0RY3+SxWyoTBiiNt1PH26z7EZgY7Yos0HiKxbMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkrrtgLpC0mstnU4XtLx2v6T9kvZJ+mSvCjdrim47oAI82NLp9CkASdcAi4Bryzr/NNq4xGxYddUB9RwWAhtK66wfAfuB6xP1mTVe5jPRstLQfq2kKWVsKvBqyzIjZews7oBqw6LbEK0GrgbmUHU9XVnGNc6y43Y3jYg1ETE3IuZOnjy5yzLM+q+rEEXE0Yg4HRFngIf4+SHbCDC9ZdFpwOFciWbN1m0H1Ctbnt4OjJ652wQsknSRpJlUHVC/nyvRrNm67YD6cUlzqA7VDgF3AUTEHkmPAz+kanR/T0Sc7k3pZs1QawfUsvyXgC9lijIbJL5iwSzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwis6Rumzc+1tK48ZCknWV8hqSftrz2tV4Wb9YEE36zlap54z8Aj4wORMQfjT6WtBJ4s2X5AxExp64CzZquk6+HPyNpxnivSRLwWeD36y3LbHBkPxPdCByNiFdaxmZK+oGk70m6Mfn+Zo3XyeHcuSwG1rc8PwJcFRHHJX0M+JakayPixNgVJS0FlgJMmTJl7MtmA6PrmUjSBcAfAo+NjpUe3MfL4x3AAeBD463vDqg2LDKHc38AvBQRI6MDki4b/RUISbOomjcezJVo1mydnOJeD/wX8GFJI5LuLC8t4t2HcgA3AbskvQD8K3B3RHT6ixJmA6nb5o1ExJ+MM7YR2Jgvy2xw+IoFsySHyCzJITJLcojMkhwisySHyCzJITJLcojMkhwis6TsVdy1eHPSGf7t4v/tdxk2jm3z56fWn/f00zVVUr/f2by5lvfxTGSW5BCZJTlEZkmN+ExkzdXkzzRN4ZnILMkzkb1n1TXLKiJqeaNUEVL/izA7246ImDvRQp18PXy6pO9K2itpj6TPl/FLJG2R9Eq5n1LGJemrkvZL2iXpuvy+mDVXJ5+JTgH3RcSvA/OAeyRdAywHtkbEbGBreQ5wK1WDktlULbFW1161WYNMGKKIOBIRz5fHbwF7ganAQmBdWWwdcFt5vBB4JCrbgIslXVl75WYNcV5n50o74Y8CzwJXRMQRqIIGXF4Wmwq82rLaSBkzG0odn52T9H6qTj5fiIgTVRvu8RcdZ+ysEwetHVDNBllHM5Gk91EF6NGI+GYZPjp6mFbuj5XxEWB6y+rTgMNj37O1A2q3xZs1QSdn5wQ8DOyNiFUtL20ClpTHS4AnW8bvKGfp5gFvjh72mQ2liDjnDfhdqsOxXcDOclsAfJDqrNwr5f6SsryAf6Tqw/0iMLeDbYRvvjXwtn2iv7sR4f9sNTuHev6z1czOzSEyS3KIzJIcIrMkh8gsqSnfJ3odOFnuh8WlDM/+DNO+QOf786udvFkjTnEDSNo+TFcvDNP+DNO+QP3748M5sySHyCypSSFa0+8CajZM+zNM+wI1709jPhOZDaomzURmA6nvIZI0X9K+0thk+cRrNI+kQ5JelLRT0vYyNm4jlyaStFbSMUm7W8YGthFNm/1ZIem18me0U9KCltfuL/uzT9Inz3uDnVzq3asbMInqKxOzgAuBF4Br+llTl/txCLh0zNiXgeXl8XLg7/pd5znqvwm4Dtg9Uf1UX4P5NtVXXuYBz/a7/g73ZwXwF+Mse035e3cRMLP8fZx0Ptvr90x0PbA/Ig5GxDvABqpGJ8OgXSOXxomIZ4A3xgwPbCOaNvvTzkJgQ0T8LCJ+BOyn+nvZsX6HaFiamgSwWdKO0jsC2jdyGRTD2IhmWTkEXdtyeJ3en36HqKOmJgPghoi4jqrn3j2Sbup3QT00qH9mq4GrgTnAEWBlGU/vT79D1FFTk6aLiMPl/hjwBNXhQLtGLoMi1YimaSLiaEScjogzwEP8/JAtvT/9DtFzwGxJMyVdCCyianQyMCRNlvSB0cfALcBu2jdyGRRD1YhmzOe226n+jKDan0WSLpI0k6pz7/fP680bcCZlAfAy1VmRB/pdTxf1z6I6u/MCsGd0H2jTyKWJN2A91SHO/1H9y3xnu/rpohFNQ/bnn0u9u0pwrmxZ/oGyP/uAW893e75iwSyp34dzZgPPITJLcojMkhwisySHyCzJITJLcojMkhwis6T/BzF6WOXJ/icoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 初期状態を描画してみる\n",
    "observation = env.reset()  # 環境の初期化\n",
    "plt.imshow(observation)  # 描画してみましょう\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 動画の描画関数の宣言\n",
    "# 参考URL http://nbviewer.jupyter.org/github/patrickmineault\n",
    "# /xcorr-notebooks/blob/master/Render%20OpenAI%20gym%20as%20GIF.ipynb\n",
    "from JSAnimation.IPython_display import display_animation\n",
    "from matplotlib import animation\n",
    "from IPython.display import display\n",
    "\n",
    "\n",
    "def display_frames_as_gif(frames):\n",
    "    \"\"\"\n",
    "    Displays a list of frames as a gif, with controls\n",
    "    \"\"\"\n",
    "    plt.figure(figsize=(frames[0].shape[1]/72.0, frames[0].shape[0]/72.0),\n",
    "               dpi=72)\n",
    "    patch = plt.imshow(frames[0])\n",
    "    plt.axis('off')\n",
    "\n",
    "    def animate(i):\n",
    "        patch.set_data(frames[i])\n",
    "\n",
    "    anim = animation.FuncAnimation(plt.gcf(), animate, frames=len(frames),\n",
    "                                   interval=50)\n",
    "\n",
    "    anim.save('breakout.mp4')  # 動画のファイル名と保存です\n",
    "    display(display_animation(anim, default_mode='loop'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Define the Animation class */\n",
       "  function Animation(frames, img_id, slider_id, interval, loop_select_id){\n",
       "    this.img_id = img_id;\n",
       "    this.slider_id = slider_id;\n",
       "    this.loop_select_id = loop_select_id;\n",
       "    this.interval = interval;\n",
       "    this.current_frame = 0;\n",
       "    this.direction = 0;\n",
       "    this.timer = null;\n",
       "    this.frames = new Array(frames.length);\n",
       "\n",
       "    for (var i=0; i<frames.length; i++)\n",
       "    {\n",
       "     this.frames[i] = new Image();\n",
       "     this.frames[i].src = frames[i];\n",
       "    }\n",
       "    document.getElementById(this.slider_id).max = this.frames.length - 1;\n",
       "    this.set_frame(this.current_frame);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.get_loop_state = function(){\n",
       "    var button_group = document[this.loop_select_id].state;\n",
       "    for (var i = 0; i < button_group.length; i++) {\n",
       "        var button = button_group[i];\n",
       "        if (button.checked) {\n",
       "            return button.value;\n",
       "        }\n",
       "    }\n",
       "    return undefined;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.set_frame = function(frame){\n",
       "    this.current_frame = frame;\n",
       "    document.getElementById(this.img_id).src = this.frames[this.current_frame].src;\n",
       "    document.getElementById(this.slider_id).value = this.current_frame;\n",
       "  }\n",
       "\n",
       "  Animation.prototype.next_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.min(this.frames.length - 1, this.current_frame + 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.previous_frame = function()\n",
       "  {\n",
       "    this.set_frame(Math.max(0, this.current_frame - 1));\n",
       "  }\n",
       "\n",
       "  Animation.prototype.first_frame = function()\n",
       "  {\n",
       "    this.set_frame(0);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.last_frame = function()\n",
       "  {\n",
       "    this.set_frame(this.frames.length - 1);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.slower = function()\n",
       "  {\n",
       "    this.interval /= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.faster = function()\n",
       "  {\n",
       "    this.interval *= 0.7;\n",
       "    if(this.direction > 0){this.play_animation();}\n",
       "    else if(this.direction < 0){this.reverse_animation();}\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_forward = function()\n",
       "  {\n",
       "    this.current_frame += 1;\n",
       "    if(this.current_frame < this.frames.length){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.first_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.last_frame();\n",
       "        this.reverse_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.last_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.anim_step_reverse = function()\n",
       "  {\n",
       "    this.current_frame -= 1;\n",
       "    if(this.current_frame >= 0){\n",
       "      this.set_frame(this.current_frame);\n",
       "    }else{\n",
       "      var loop_state = this.get_loop_state();\n",
       "      if(loop_state == \"loop\"){\n",
       "        this.last_frame();\n",
       "      }else if(loop_state == \"reflect\"){\n",
       "        this.first_frame();\n",
       "        this.play_animation();\n",
       "      }else{\n",
       "        this.pause_animation();\n",
       "        this.first_frame();\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.pause_animation = function()\n",
       "  {\n",
       "    this.direction = 0;\n",
       "    if (this.timer){\n",
       "      clearInterval(this.timer);\n",
       "      this.timer = null;\n",
       "    }\n",
       "  }\n",
       "\n",
       "  Animation.prototype.play_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = 1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_forward();}, this.interval);\n",
       "  }\n",
       "\n",
       "  Animation.prototype.reverse_animation = function()\n",
       "  {\n",
       "    this.pause_animation();\n",
       "    this.direction = -1;\n",
       "    var t = this;\n",
       "    if (!this.timer) this.timer = setInterval(function(){t.anim_step_reverse();}, this.interval);\n",
       "  }\n",
       "</script>\n",
       "\n",
       "<div class=\"animation\" align=\"center\">\n",
       "    <img id=\"_anim_imgYGVDCZYNGRVDHKNU\">\n",
       "    <br>\n",
       "    <input id=\"_anim_sliderYGVDCZYNGRVDHKNU\" type=\"range\" style=\"width:350px\" name=\"points\" min=\"0\" max=\"1\" step=\"1\" value=\"0\" onchange=\"animYGVDCZYNGRVDHKNU.set_frame(parseInt(this.value));\"></input>\n",
       "    <br>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.slower()\">&#8211;</button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.first_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.previous_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.reverse_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.pause_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.play_animation()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.next_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.last_frame()\"><img class=\"anim_icon\" src=\"\"></button>\n",
       "    <button onclick=\"animYGVDCZYNGRVDHKNU.faster()\">+</button>\n",
       "  <form action=\"#n\" name=\"_anim_loop_selectYGVDCZYNGRVDHKNU\" class=\"anim_control\">\n",
       "    <input type=\"radio\" name=\"state\" value=\"once\" > Once </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"loop\" checked> Loop </input>\n",
       "    <input type=\"radio\" name=\"state\" value=\"reflect\" > Reflect </input>\n",
       "  </form>\n",
       "</div>\n",
       "\n",
       "\n",
       "<script language=\"javascript\">\n",
       "  /* Instantiate the Animation class. */\n",
       "  /* The IDs given should match those used in the template above. */\n",
       "  (function() {\n",
       "    var img_id = \"_anim_imgYGVDCZYNGRVDHKNU\";\n",
       "    var slider_id = \"_anim_sliderYGVDCZYNGRVDHKNU\";\n",
       "    var loop_select_id = \"_anim_loop_selectYGVDCZYNGRVDHKNU\";\n",
       "    var frames = new Array(0);\n",
       "    \n",
       "  frames[0] = \"\"\n",
       "  frames[1] = \"\"\n",
       "  frames[2] = \"\"\n",
       "  frames[3] = \"\"\n",
       "  frames[4] = \"\"\n",
       "  frames[5] = \"\"\n",
       "  frames[6] = \"\"\n",
       "  frames[7] = \"\"\n",
       "  frames[8] = \"\"\n",
       "  frames[9] = \"\"\n",
       "  frames[10] = \"\"\n",
       "  frames[11] = \"\"\n",
       "  frames[12] = \"\"\n",
       "  frames[13] = \"\"\n",
       "  frames[14] = \"\"\n",
       "  frames[15] = \"\"\n",
       "  frames[16] = \"\"\n",
       "  frames[17] = \"\"\n",
       "  frames[18] = \"\"\n",
       "  frames[19] = \"\"\n",
       "  frames[20] = \"\"\n",
       "  frames[21] = \"\"\n",
       "  frames[22] = \"\"\n",
       "  frames[23] = \"\"\n",
       "  frames[24] = \"\"\n",
       "  frames[25] = \"\"\n",
       "  frames[26] = \"\"\n",
       "  frames[27] = \"\"\n",
       "  frames[28] = \"\"\n",
       "  frames[29] = \"\"\n",
       "  frames[30] = \"\"\n",
       "  frames[31] = \"\"\n",
       "  frames[32] = \"\"\n",
       "  frames[33] = \"\"\n",
       "  frames[34] = \"\"\n",
       "  frames[35] = \"\"\n",
       "  frames[36] = \"\"\n",
       "  frames[37] = \"\"\n",
       "  frames[38] = \"\"\n",
       "  frames[39] = \"\"\n",
       "  frames[40] = \"\"\n",
       "  frames[41] = \"\"\n",
       "  frames[42] = \"\"\n",
       "  frames[43] = \"\"\n",
       "  frames[44] = \"\"\n",
       "  frames[45] = \"\"\n",
       "  frames[46] = \"\"\n",
       "  frames[47] = \"\"\n",
       "  frames[48] = \"\"\n",
       "  frames[49] = \"\"\n",
       "  frames[50] = \"\"\n",
       "  frames[51] = \"\"\n",
       "  frames[52] = \"\"\n",
       "  frames[53] = \"\"\n",
       "  frames[54] = \"\"\n",
       "  frames[55] = \"\"\n",
       "  frames[56] = \"\"\n",
       "  frames[57] = \"\"\n",
       "  frames[58] = \"\"\n",
       "  frames[59] = \"\"\n",
       "  frames[60] = \"\"\n",
       "  frames[61] = \"\"\n",
       "  frames[62] = \"\"\n",
       "  frames[63] = \"\"\n",
       "  frames[64] = \"\"\n",
       "  frames[65] = \"\"\n",
       "  frames[66] = \"\"\n",
       "  frames[67] = \"\"\n",
       "  frames[68] = \"\"\n",
       "  frames[69] = \"\"\n",
       "  frames[70] = \"\"\n",
       "  frames[71] = \"\"\n",
       "  frames[72] = \"\"\n",
       "  frames[73] = \"\"\n",
       "  frames[74] = \"\"\n",
       "  frames[75] = \"\"\n",
       "  frames[76] = \"\"\n",
       "  frames[77] = \"\"\n",
       "  frames[78] = \"\"\n",
       "  frames[79] = \"\"\n",
       "  frames[80] = \"\"\n",
       "  frames[81] = \"\"\n",
       "  frames[82] = \"\"\n",
       "  frames[83] = \"\"\n",
       "  frames[84] = \"\"\n",
       "  frames[85] = \"\"\n",
       "  frames[86] = \"\"\n",
       "  frames[87] = \"\"\n",
       "  frames[88] = \"\"\n",
       "  frames[89] = \"\"\n",
       "  frames[90] = \"\"\n",
       "  frames[91] = \"\"\n",
       "  frames[92] = \"\"\n",
       "  frames[93] = \"\"\n",
       "  frames[94] = \"\"\n",
       "  frames[95] = \"\"\n",
       "  frames[96] = \"\"\n",
       "  frames[97] = \"\"\n",
       "  frames[98] = \"\"\n",
       "  frames[99] = \"\"\n",
       "  frames[100] = \"\"\n",
       "  frames[101] = \"\"\n",
       "  frames[102] = \"\"\n",
       "  frames[103] = \"\"\n",
       "  frames[104] = \"\"\n",
       "  frames[105] = \"\"\n",
       "  frames[106] = \"\"\n",
       "  frames[107] = \"\"\n",
       "  frames[108] = \"\"\n",
       "  frames[109] = \"\"\n",
       "  frames[110] = \"\"\n",
       "  frames[111] = \"\"\n",
       "  frames[112] = \"\"\n",
       "  frames[113] = \"\"\n",
       "  frames[114] = \"\"\n",
       "  frames[115] = \"\"\n",
       "  frames[116] = \"\"\n",
       "  frames[117] = \"\"\n",
       "  frames[118] = \"\"\n",
       "  frames[119] = \"\"\n",
       "  frames[120] = \"\"\n",
       "  frames[121] = \"\"\n",
       "  frames[122] = \"\"\n",
       "  frames[123] = \"\"\n",
       "  frames[124] = \"\"\n",
       "  frames[125] = \"\"\n",
       "  frames[126] = \"\"\n",
       "  frames[127] = \"\"\n",
       "  frames[128] = \"\"\n",
       "  frames[129] = \"\"\n",
       "  frames[130] = \"\"\n",
       "  frames[131] = \"\"\n",
       "  frames[132] = \"\"\n",
       "  frames[133] = \"\"\n",
       "  frames[134] = \"\"\n",
       "  frames[135] = \"\"\n",
       "  frames[136] = \"\"\n",
       "  frames[137] = \"\"\n",
       "  frames[138] = \"\"\n",
       "  frames[139] = \"\"\n",
       "  frames[140] = \"\"\n",
       "  frames[141] = \"\"\n",
       "  frames[142] = \"\"\n",
       "  frames[143] = \"\"\n",
       "  frames[144] = \"\"\n",
       "  frames[145] = \"\"\n",
       "  frames[146] = \"\"\n",
       "  frames[147] = \"\"\n",
       "  frames[148] = \"\"\n",
       "  frames[149] = \"\"\n",
       "  frames[150] = \"\"\n",
       "  frames[151] = \"\"\n",
       "  frames[152] = \"\"\n",
       "  frames[153] = \"\"\n",
       "  frames[154] = \"\"\n",
       "  frames[155] = \"\"\n",
       "  frames[156] = \"\"\n",
       "  frames[157] = \"\"\n",
       "  frames[158] = \"\"\n",
       "  frames[159] = \"\"\n",
       "  frames[160] = \"\"\n",
       "  frames[161] = \"\"\n",
       "  frames[162] = \"\"\n",
       "  frames[163] = \"\"\n",
       "  frames[164] = \"\"\n",
       "  frames[165] = \"\"\n",
       "  frames[166] = \"\"\n",
       "  frames[167] = \"\"\n",
       "  frames[168] = \"\"\n",
       "  frames[169] = \"\"\n",
       "  frames[170] = \"\"\n",
       "  frames[171] = \"\"\n",
       "  frames[172] = \"\"\n",
       "  frames[173] = \"\"\n",
       "  frames[174] = \"\"\n",
       "  frames[175] = \"\"\n",
       "  frames[176] = \"\"\n",
       "  frames[177] = \"\"\n",
       "  frames[178] = \"\"\n",
       "  frames[179] = \"\"\n",
       "  frames[180] = \"\"\n",
       "  frames[181] = \"\"\n",
       "  frames[182] = \"\"\n",
       "  frames[183] = \"\"\n",
       "  frames[184] = \"\"\n",
       "  frames[185] = \"\"\n",
       "  frames[186] = \"\"\n",
       "  frames[187] = \"\"\n",
       "  frames[188] = \"\"\n",
       "  frames[189] = \"\"\n",
       "  frames[190] = \"\"\n",
       "  frames[191] = \"\"\n",
       "  frames[192] = \"\"\n",
       "  frames[193] = \"\"\n",
       "  frames[194] = \"\"\n",
       "  frames[195] = \"\"\n",
       "  frames[196] = \"\"\n",
       "  frames[197] = \"\"\n",
       "  frames[198] = \"\"\n",
       "  frames[199] = \"\"\n",
       "  frames[200] = \"\"\n",
       "  frames[201] = \"\"\n",
       "  frames[202] = \"\"\n",
       "  frames[203] = \"\"\n",
       "  frames[204] = \"\"\n",
       "  frames[205] = \"\"\n",
       "  frames[206] = \"\"\n",
       "  frames[207] = \"\"\n",
       "  frames[208] = \"\"\n",
       "  frames[209] = \"\"\n",
       "  frames[210] = \"\"\n",
       "  frames[211] = \"\"\n",
       "  frames[212] = \"\"\n",
       "  frames[213] = \"\"\n",
       "  frames[214] = \"\"\n",
       "  frames[215] = \"\"\n",
       "  frames[216] = \"\"\n",
       "  frames[217] = \"\"\n",
       "  frames[218] = \"\"\n",
       "  frames[219] = \"\"\n",
       "  frames[220] = \"\"\n",
       "  frames[221] = \"\"\n",
       "  frames[222] = \"\"\n",
       "  frames[223] = \"\"\n",
       "  frames[224] = \"\"\n",
       "  frames[225] = \"\"\n",
       "  frames[226] = \"\"\n",
       "  frames[227] = \"\"\n",
       "  frames[228] = \"\"\n",
       "  frames[229] = \"\"\n",
       "  frames[230] = \"\"\n",
       "  frames[231] = \"\"\n",
       "  frames[232] = \"\"\n",
       "  frames[233] = \"\"\n",
       "  frames[234] = \"\"\n",
       "  frames[235] = \"\"\n",
       "  frames[236] = \"\"\n",
       "  frames[237] = \"\"\n",
       "  frames[238] = \"\"\n",
       "  frames[239] = \"\"\n",
       "  frames[240] = \"\"\n",
       "  frames[241] = \"\"\n",
       "  frames[242] = \"\"\n",
       "  frames[243] = \"\"\n",
       "  frames[244] = \"\"\n",
       "  frames[245] = \"\"\n",
       "  frames[246] = \"\"\n",
       "  frames[247] = \"\"\n",
       "  frames[248] = \"\"\n",
       "  frames[249] = \"\"\n",
       "  frames[250] = \"\"\n",
       "  frames[251] = \"\"\n",
       "  frames[252] = \"\"\n",
       "  frames[253] = \"\"\n",
       "  frames[254] = \"\"\n",
       "  frames[255] = \"\"\n",
       "  frames[256] = \"\"\n",
       "  frames[257] = \"\"\n",
       "  frames[258] = \"\"\n",
       "  frames[259] = \"\"\n",
       "  frames[260] = \"\"\n",
       "  frames[261] = \"\"\n",
       "  frames[262] = \"\"\n",
       "  frames[263] = \"\"\n",
       "  frames[264] = \"\"\n",
       "  frames[265] = \"\"\n",
       "  frames[266] = \"\"\n",
       "  frames[267] = \"\"\n",
       "  frames[268] = \"\"\n",
       "  frames[269] = \"\"\n",
       "  frames[270] = \"\"\n",
       "  frames[271] = \"\"\n",
       "  frames[272] = \"\"\n",
       "  frames[273] = \"\"\n",
       "  frames[274] = \"\"\n",
       "  frames[275] = \"\"\n",
       "  frames[276] = \"\"\n",
       "  frames[277] = \"\"\n",
       "  frames[278] = \"\"\n",
       "  frames[279] = \"\"\n",
       "  frames[280] = \"\"\n",
       "  frames[281] = \"\"\n",
       "  frames[282] = \"\"\n",
       "  frames[283] = \"\"\n",
       "  frames[284] = \"\"\n",
       "  frames[285] = \"\"\n",
       "  frames[286] = \"\"\n",
       "  frames[287] = \"\"\n",
       "  frames[288] = \"\"\n",
       "  frames[289] = \"\"\n",
       "  frames[290] = \"\"\n",
       "  frames[291] = \"\"\n",
       "  frames[292] = \"\"\n",
       "  frames[293] = \"\"\n",
       "  frames[294] = \"\"\n",
       "  frames[295] = \"\"\n",
       "  frames[296] = \"\"\n",
       "  frames[297] = \"\"\n",
       "  frames[298] = \"\"\n",
       "  frames[299] = \"\"\n",
       "  frames[300] = \"\"\n",
       "  frames[301] = \"\"\n",
       "  frames[302] = \"\"\n",
       "  frames[303] = \"\"\n",
       "  frames[304] = \"\"\n",
       "  frames[305] = \"\"\n",
       "  frames[306] = \"\"\n",
       "  frames[307] = \"\"\n",
       "  frames[308] = \"\"\n",
       "  frames[309] = \"\"\n",
       "  frames[310] = \"\"\n",
       "  frames[311] = \"\"\n",
       "  frames[312] = \"\"\n",
       "  frames[313] = \"\"\n",
       "  frames[314] = \"\"\n",
       "  frames[315] = \"\"\n",
       "  frames[316] = \"\"\n",
       "  frames[317] = \"\"\n",
       "  frames[318] = \"\"\n",
       "  frames[319] = \"\"\n",
       "  frames[320] = \"\"\n",
       "  frames[321] = \"\"\n",
       "  frames[322] = \"\"\n",
       "  frames[323] = \"\"\n",
       "  frames[324] = \"\"\n",
       "  frames[325] = \"\"\n",
       "  frames[326] = \"\"\n",
       "  frames[327] = \"\"\n",
       "  frames[328] = \"\"\n",
       "  frames[329] = \"\"\n",
       "  frames[330] = \"\"\n",
       "  frames[331] = \"\"\n",
       "  frames[332] = \"\"\n",
       "  frames[333] = \"\"\n",
       "  frames[334] = \"\"\n",
       "  frames[335] = \"\"\n",
       "  frames[336] = \"\"\n",
       "  frames[337] = \"\"\n",
       "  frames[338] = \"\"\n",
       "  frames[339] = \"\"\n",
       "  frames[340] = \"\"\n",
       "  frames[341] = \"\"\n",
       "  frames[342] = \"\"\n",
       "  frames[343] = \"\"\n",
       "  frames[344] = \"\"\n",
       "  frames[345] = \"\"\n",
       "  frames[346] = \"\"\n",
       "  frames[347] = \"\"\n",
       "  frames[348] = \"\"\n",
       "  frames[349] = \"\"\n",
       "  frames[350] = \"\"\n",
       "  frames[351] = \"\"\n",
       "  frames[352] = \"\"\n",
       "  frames[353] = \"\"\n",
       "  frames[354] = \"\"\n",
       "  frames[355] = \"\"\n",
       "  frames[356] = \"\"\n",
       "  frames[357] = \"\"\n",
       "  frames[358] = \"\"\n",
       "  frames[359] = \"\"\n",
       "  frames[360] = \"\"\n",
       "  frames[361] = \"\"\n",
       "  frames[362] = \"\"\n",
       "  frames[363] = \"\"\n",
       "  frames[364] = \"\"\n",
       "  frames[365] = \"\"\n",
       "  frames[366] = \"\"\n",
       "  frames[367] = \"\"\n",
       "  frames[368] = \"\"\n",
       "  frames[369] = \"\"\n",
       "  frames[370] = \"\"\n",
       "  frames[371] = \"\"\n",
       "  frames[372] = \"\"\n",
       "  frames[373] = \"\"\n",
       "  frames[374] = \"\"\n",
       "  frames[375] = \"\"\n",
       "  frames[376] = \"\"\n",
       "  frames[377] = \"\"\n",
       "  frames[378] = \"\"\n",
       "  frames[379] = \"\"\n",
       "  frames[380] = \"\"\n",
       "  frames[381] = \"\"\n",
       "  frames[382] = \"\"\n",
       "  frames[383] = \"\"\n",
       "  frames[384] = \"\"\n",
       "  frames[385] = \"\"\n",
       "  frames[386] = \"\"\n",
       "  frames[387] = \"\"\n",
       "  frames[388] = \"\"\n",
       "  frames[389] = \"\"\n",
       "  frames[390] = \"\"\n",
       "  frames[391] = \"\"\n",
       "  frames[392] = \"\"\n",
       "  frames[393] = \"\"\n",
       "  frames[394] = \"\"\n",
       "  frames[395] = \"\"\n",
       "  frames[396] = \"\"\n",
       "  frames[397] = \"\"\n",
       "  frames[398] = \"\"\n",
       "  frames[399] = \"\"\n",
       "  frames[400] = \"\"\n",
       "  frames[401] = \"\"\n",
       "  frames[402] = \"\"\n",
       "  frames[403] = \"\"\n",
       "  frames[404] = \"\"\n",
       "  frames[405] = \"\"\n",
       "  frames[406] = \"\"\n",
       "  frames[407] = \"\"\n",
       "  frames[408] = \"\"\n",
       "  frames[409] = \"\"\n",
       "  frames[410] = \"\"\n",
       "  frames[411] = \"\"\n",
       "  frames[412] = \"\"\n",
       "  frames[413] = \"\"\n",
       "  frames[414] = \"\"\n",
       "  frames[415] = \"\"\n",
       "  frames[416] = \"\"\n",
       "  frames[417] = \"\"\n",
       "  frames[418] = \"\"\n",
       "  frames[419] = \"\"\n",
       "  frames[420] = \"\"\n",
       "  frames[421] = \"\"\n",
       "  frames[422] = \"\"\n",
       "  frames[423] = \"\"\n",
       "  frames[424] = \"\"\n",
       "  frames[425] = \"\"\n",
       "  frames[426] = \"\"\n",
       "  frames[427] = \"\"\n",
       "  frames[428] = \"\"\n",
       "  frames[429] = \"\"\n",
       "  frames[430] = \"\"\n",
       "  frames[431] = \"\"\n",
       "  frames[432] = \"\"\n",
       "  frames[433] = \"\"\n",
       "  frames[434] = \"\"\n",
       "  frames[435] = \"\"\n",
       "  frames[436] = \"\"\n",
       "  frames[437] = \"\"\n",
       "  frames[438] = \"\"\n",
       "  frames[439] = \"\"\n",
       "  frames[440] = \"\"\n",
       "  frames[441] = \"\"\n",
       "  frames[442] = \"\"\n",
       "  frames[443] = \"\"\n",
       "  frames[444] = \"\"\n",
       "  frames[445] = \"\"\n",
       "  frames[446] = \"\"\n",
       "  frames[447] = \"\"\n",
       "  frames[448] = \"\"\n",
       "  frames[449] = \"\"\n",
       "  frames[450] = \"\"\n",
       "  frames[451] = \"\"\n",
       "  frames[452] = \"\"\n",
       "  frames[453] = \"\"\n",
       "  frames[454] = \"\"\n",
       "  frames[455] = \"\"\n",
       "  frames[456] = \"\"\n",
       "  frames[457] = \"\"\n",
       "  frames[458] = \"\"\n",
       "  frames[459] = \"\"\n",
       "  frames[460] = \"\"\n",
       "  frames[461] = \"\"\n",
       "  frames[462] = \"\"\n",
       "  frames[463] = \"\"\n",
       "  frames[464] = \"\"\n",
       "  frames[465] = \"\"\n",
       "  frames[466] = \"\"\n",
       "  frames[467] = \"\"\n",
       "  frames[468] = \"\"\n",
       "  frames[469] = \"\"\n",
       "  frames[470] = \"\"\n",
       "  frames[471] = \"\"\n",
       "  frames[472] = \"\"\n",
       "  frames[473] = \"\"\n",
       "  frames[474] = \"\"\n",
       "\n",
       "\n",
       "    /* set a timeout to make sure all the above elements are created before\n",
       "       the object is initialized. */\n",
       "    setTimeout(function() {\n",
       "        animYGVDCZYNGRVDHKNU = new Animation(frames, img_id, slider_id, 50, loop_select_id);\n",
       "    }, 0);\n",
       "  })()\n",
       "</script>\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 適当に動かしてみましょう\n",
    "\n",
    "frames = []  # 画像を格納していく変数\n",
    "observation = env.reset()  # 環境の初期化\n",
    "\n",
    "for step in range(1000):  # 最大1000エピソードのループ\n",
    "    frames.append(observation)  # 変換せずに画像を保存\n",
    "    action = np.random.randint(0, 4)  # 0～3のランダムな行動を求める\n",
    "    observation_next, reward, done, info = env.step(action)  # 実行\n",
    "\n",
    "    observation = observation_next  # 状態の更新\n",
    "\n",
    "    if done:  # 終了したらループから抜ける\n",
    "        break\n",
    "\n",
    "display_frames_as_gif(frames)  # 動画を保存と描画してみよう\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 以上でBreakoutをランダムに動かすことができました。\n",
    "# つづいて強化学習を適用していきましょう。\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
